In [1]:
import numpy as np
import pandas as pd
import json
import sys
import os
import matplotlib
matplotlib.use('Agg') 
import matplotlib.pyplot as plt
import seaborn as sns
import pdb

from util import utils as data_utils

%pylab inline
%matplotlib inline
plt.rcParams['figure.figsize'] = (10.0, 8.0) # set default size of plots
plt.rcParams['image.interpolation'] = 'nearest'
plt.rcParams['image.cmap'] = 'Blues'

# for auto-reloading external modules
# see http://stackoverflow.com/questions/1907993/autoreload-of-modules-in-ipython
%load_ext autoreload
%autoreload 2

json_file = './cifar_results/no_noise/vgg_bn_arch_lr_001/checkpoint_200.json'
FDIR = os.path.dirname(json_file)
NUM_CLASSIFY = 10
Populating the interactive namespace from numpy and matplotlib
In [ ]:
 
In [2]:
# Plot gradients norms for the entire learning process
grads_json_filename = os.path.join(FDIR, 'model_grads.json')
grads = [[], [], []]
grads_key = ['max_grad_w1_16', 'max_grad_w1_32', 'max_grad_w1_64']
if os.path.exists(grads_json_filename):
    with open(grads_json_filename, 'r') as fp:
        data = json.load(fp)
        for i, k in enumerate(grads_key):
            if data[0].get(k, None) is None:
                continue
            for batch_grads in data:
                grads[i].append(batch_grads[k])

def plot_grads(grads, title, x_label, y_label, figsize=(10, 8)):
    plt.figure(figsize=figsize)
    # plt.subplot(2, 1, 1)
    plt.plot(grads)
    plt.title(title)
    plt.ylabel(y_label)
    plt.xlabel(x_label)
    
for i, g in enumerate(grads):
    if len(g) > 0:
        plot_grads(g, grads_key[i], 'iterations', grads_key[i])
        # pass
In [3]:
with open(json_file, 'r') as fp:
    data = json.load(fp)
# Loss history might not be of equal length.
train_loss_hist = data['train_loss_history']
val_loss_hist = data['val_loss_history']

# pdb.set_trace()
def plot_loss_hist(loss_hist, title,):
    plt.figure(figsize=(5,4))
    plt.subplot(1, 1, 1)
    plt.plot(loss_hist)
    plt.title(title)  # Train Loss
    plt.ylabel('loss')
    plt.xlabel('time')
    plt.show()
    
plot_loss_hist(train_loss_hist, 'Train Loss')
plot_loss_hist(val_loss_hist, 'Val loss')

if data.get('crit1_loss_history', None) is not None:
    plot_loss_hist(data['crit1_loss_history'], 'Target criterion loss')

if data.get('crit2_loss_history', None) is not None and \
    len(data['crit2_loss_history']) > 0:
    plot_loss_hist(data['crit2_loss_history'], 'Pred criterion loss')

if data.get('pred_loss_history', None) is not None and \
    len(data['pred_loss_history']) > 0:
    plot_loss_hist(data['pred_loss_history'], 'Total Pred loss (beta*t + (1-beta)*p)')    

if data.get('beta_loss_history', None) is not None and \
    len(data['beta_loss_history']) > 0:
    plot_loss_hist(data['beta_loss_history'], 'Beta loss')
In [4]:
if data.get('KL_loss_history', None) is not None:
    # Loss history might not be of equal length.
    KL_loss_hist = data['KL_loss_history']

    plt.figure(figsize=(10,8))
    plt.plot(KL_loss_hist)
    plt.title('KL loss')
    plt.ylabel('loss')
    plt.xlabel('time')
    plt.show()
In [5]:
def get_conf(json_file, num_classes=26, json_key='conf'):
    with open(json_file, 'r') as fp:
        data = json.load(fp)
        conf = data.get(json_key, None)
    if conf is None:
        return
    # c1 = conf.split('\n')[1].split("]")[0].split("[ ")[1].split(" ")
    c1 = conf.split('\n')
    # print(c1)
    conf_mat, row_idx = np.zeros((num_classes, num_classes)), 0
    for i in c1:
        #pdb.set_trace()
        is_conf_row = False
        if ']' in i and '[[' in i:
            val = i.split(']')[0].split('[[')[1].split(' ')
            is_conf_row = True
        elif ']' in i and '[' in i:
            val = i.split(']')[0].split('[')[1].split(' ')
            is_conf_row = True
        if is_conf_row:
            col_idx = 0
            for v in val:
                if not len(v):
                    continue
                try:
                    conf_mat[row_idx, col_idx] = int(v)
                    col_idx = col_idx + 1
                except:
                    continue
            row_idx = row_idx + 1
    
    assert(row_idx == num_classes)
    conf_mat = conf_mat.astype(int)
    fdir = os.path.dirname(json_file)
    json_name = os.path.basename(json_file)[:-5]
    conf_file_name = fdir + '/' + 'conf_' + json_name + '.txt'
    np.savetxt(conf_file_name, conf_mat, fmt='%d', delimiter=', ')
    return conf_mat


def plot_conf(norm_conf):
  # Plot using seaborn
  # (this is style I used for ResNet matrix)
  plt.figure(figsize=(10,6))
  df_cm = pd.DataFrame(norm_conf)
  sns.heatmap(df_cm, annot=True, cmap="Blues")
  plt.show()
In [6]:
def get_sorted_checkpoints(fdir):
    # Checkpoint files are named as 'checkpoint_%d.json'
    checkpoint_map = {}
    for f in os.listdir(fdir):
        if f.endswith('json') and f.startswith('checkpoint'):
            checkpoint_num = int(f.split('checkpoint_')[-1].split('.')[0])
            checkpoint_map[checkpoint_num] = f
    sorted_checkpoints = []
    for k in sorted(checkpoint_map.keys()):
        v = checkpoint_map[k]
        sorted_checkpoints.append(v)
    return sorted_checkpoints
In [7]:
def best_f_scores(fdir, num_classes=5): 
    best_checkpoints = [None, None, None]
    best_3_fscores = [0, 0, 0]
    best_confs = [np.array(()), np.array(()), np.array(())]
    f1_weight_list = [1.0] * num_classes
    f1_weights = np.array(f1_weight_list)
    sorted_checkpoint_files = get_sorted_checkpoints(fdir)
    for f in sorted_checkpoint_files:
        json_file = fdir + '/' + f
        conf = get_conf(json_file, num_classes, json_key='val_conf')
        norm_conf = data_utils.normalize_conf(conf)
        f1 = data_utils.get_f1_score(conf, f1_weights)
        kappa = data_utils.computeKappa(conf)
        wt_f1 = data_utils.computeWeightedF1(conf)
        print('file: {}, f1: {:.3f}, kappa: {:.3f}, weighted-F1: {:.3f}'.format(
                f, f1, kappa, wt_f1))
        plot_conf(norm_conf)
        max_idx = -1
        for i in range(len(best_3_fscores)):
            if best_3_fscores[i] > f1:
                break
            max_idx = i
        for j in range(max_idx):
            best_3_fscores[j] = best_3_fscores[j+1]
            best_confs[j] = best_confs[j+1]
            best_checkpoints[j] = best_checkpoints[j+1]

        best_3_fscores[max_idx] = f1
        best_confs[max_idx] = conf
        best_checkpoints[max_idx] = f

    return best_3_fscores, best_confs, best_checkpoints
In [8]:
def plot_train_conf(fdir, num_classes=5):
    sorted_checkpoint_files = get_sorted_checkpoints(fdir)
    if len(sorted_checkpoint_files) > 0:
        last_checkpoint = sorted_checkpoint_files[-1]
        json_file = fdir + '/' + last_checkpoint
        conf = get_conf(json_file, num_classes=num_classes, json_key='train_conf')
        print(conf)
        norm_conf = data_utils.normalize_conf(conf)
        f1_weight_list = [1.0] * num_classes
        f1_weights = np.array(f1_weight_list)
        f1 = data_utils.get_f1_score(conf, f1_weights)
        kappa = data_utils.computeKappa(conf)
        wt_f1 = data_utils.computeWeightedF1(conf)
        print('file: {}, f1: {:.3f}, kappa: {:.3f}, weighted-F1: {:.3f}'.format(
            f, f1, kappa, wt_f1))
        plot_conf(norm_conf)

plot_train_conf(FDIR, num_classes=NUM_CLASSIFY)
[[4694    8   67   39   21    0    4   26  105   36]
 [  38 4796    0   12    0    0   18    1   40   95]
 [ 151    0 4418  120   98   79   78   30   23    3]
 [  29   10  105 4087   77  499  115   44    4   30]
 [  31    2   84  127 4506   75   60   99    8    8]
 [   6    1   68  487   86 4224   19   95    6    8]
 [  11    7   99  110   37   36 4688    2    7    3]
 [  18    1   37   76   72  110    1 4672    3   10]
 [  71   25   21    8    4    3    8    8 4836   16]
 [  60  125    2   15    1    2    8    8   50 4729]]
file: <built-in method f of mtrand.RandomState object at 0x7f616712f5f0>, f1: 0.913, kappa: 0.921, weighted-F1: 0.913
In [9]:
best_f_scores(FDIR, num_classes=NUM_CLASSIFY)
file: checkpoint_1.json, f1: 0.178, kappa: 0.036, weighted-F1: 0.178
file: checkpoint_2.json, f1: 0.526, kappa: 0.404, weighted-F1: 0.526
file: checkpoint_3.json, f1: 0.601, kappa: 0.561, weighted-F1: 0.601
file: checkpoint_4.json, f1: 0.629, kappa: 0.584, weighted-F1: 0.629
file: checkpoint_5.json, f1: 0.661, kappa: 0.692, weighted-F1: 0.661
file: checkpoint_6.json, f1: 0.713, kappa: 0.711, weighted-F1: 0.713
file: checkpoint_7.json, f1: 0.765, kappa: 0.760, weighted-F1: 0.765
file: checkpoint_8.json, f1: 0.783, kappa: 0.779, weighted-F1: 0.783
file: checkpoint_9.json, f1: 0.788, kappa: 0.793, weighted-F1: 0.788
file: checkpoint_10.json, f1: 0.793, kappa: 0.804, weighted-F1: 0.793
file: checkpoint_11.json, f1: 0.828, kappa: 0.830, weighted-F1: 0.828
file: checkpoint_12.json, f1: 0.788, kappa: 0.795, weighted-F1: 0.788
file: checkpoint_13.json, f1: 0.829, kappa: 0.822, weighted-F1: 0.829
file: checkpoint_14.json, f1: 0.838, kappa: 0.821, weighted-F1: 0.838
file: checkpoint_15.json, f1: 0.821, kappa: 0.817, weighted-F1: 0.821
file: checkpoint_16.json, f1: 0.828, kappa: 0.816, weighted-F1: 0.828
file: checkpoint_17.json, f1: 0.852, kappa: 0.847, weighted-F1: 0.852
file: checkpoint_18.json, f1: 0.831, kappa: 0.822, weighted-F1: 0.831
file: checkpoint_19.json, f1: 0.853, kappa: 0.850, weighted-F1: 0.853
file: checkpoint_20.json, f1: 0.859, kappa: 0.857, weighted-F1: 0.859
file: checkpoint_21.json, f1: 0.868, kappa: 0.868, weighted-F1: 0.868
file: checkpoint_22.json, f1: 0.864, kappa: 0.866, weighted-F1: 0.864
file: checkpoint_23.json, f1: 0.858, kappa: 0.834, weighted-F1: 0.858
file: checkpoint_24.json, f1: 0.857, kappa: 0.842, weighted-F1: 0.857
file: checkpoint_25.json, f1: 0.874, kappa: 0.867, weighted-F1: 0.874
file: checkpoint_26.json, f1: 0.874, kappa: 0.868, weighted-F1: 0.874
file: checkpoint_27.json, f1: 0.874, kappa: 0.870, weighted-F1: 0.874
file: checkpoint_28.json, f1: 0.877, kappa: 0.877, weighted-F1: 0.877
file: checkpoint_29.json, f1: 0.874, kappa: 0.868, weighted-F1: 0.874
file: checkpoint_30.json, f1: 0.872, kappa: 0.864, weighted-F1: 0.872
file: checkpoint_31.json, f1: 0.874, kappa: 0.867, weighted-F1: 0.874
file: checkpoint_32.json, f1: 0.875, kappa: 0.871, weighted-F1: 0.875
file: checkpoint_33.json, f1: 0.873, kappa: 0.867, weighted-F1: 0.873
file: checkpoint_34.json, f1: 0.874, kappa: 0.869, weighted-F1: 0.874
file: checkpoint_35.json, f1: 0.874, kappa: 0.868, weighted-F1: 0.874
file: checkpoint_36.json, f1: 0.874, kappa: 0.868, weighted-F1: 0.874
file: checkpoint_37.json, f1: 0.873, kappa: 0.866, weighted-F1: 0.873
file: checkpoint_38.json, f1: 0.876, kappa: 0.872, weighted-F1: 0.876
file: checkpoint_39.json, f1: 0.876, kappa: 0.872, weighted-F1: 0.876
file: checkpoint_40.json, f1: 0.873, kappa: 0.866, weighted-F1: 0.873
file: checkpoint_41.json, f1: 0.874, kappa: 0.869, weighted-F1: 0.874
file: checkpoint_42.json, f1: 0.875, kappa: 0.871, weighted-F1: 0.875
file: checkpoint_43.json, f1: 0.874, kappa: 0.871, weighted-F1: 0.874
file: checkpoint_44.json, f1: 0.874, kappa: 0.868, weighted-F1: 0.874
file: checkpoint_45.json, f1: 0.873, kappa: 0.866, weighted-F1: 0.873
file: checkpoint_46.json, f1: 0.873, kappa: 0.867, weighted-F1: 0.873
file: checkpoint_47.json, f1: 0.876, kappa: 0.872, weighted-F1: 0.876
file: checkpoint_48.json, f1: 0.875, kappa: 0.870, weighted-F1: 0.875
file: checkpoint_49.json, f1: 0.875, kappa: 0.870, weighted-F1: 0.875
file: checkpoint_50.json, f1: 0.875, kappa: 0.870, weighted-F1: 0.875
file: checkpoint_51.json, f1: 0.875, kappa: 0.869, weighted-F1: 0.875
file: checkpoint_52.json, f1: 0.874, kappa: 0.869, weighted-F1: 0.874
file: checkpoint_53.json, f1: 0.873, kappa: 0.865, weighted-F1: 0.873
file: checkpoint_54.json, f1: 0.873, kappa: 0.866, weighted-F1: 0.873
file: checkpoint_55.json, f1: 0.875, kappa: 0.871, weighted-F1: 0.875
file: checkpoint_56.json, f1: 0.873, kappa: 0.868, weighted-F1: 0.873
file: checkpoint_57.json, f1: 0.875, kappa: 0.869, weighted-F1: 0.875
file: checkpoint_58.json, f1: 0.875, kappa: 0.870, weighted-F1: 0.875
file: checkpoint_59.json, f1: 0.874, kappa: 0.867, weighted-F1: 0.874
file: checkpoint_60.json, f1: 0.875, kappa: 0.869, weighted-F1: 0.875
file: checkpoint_61.json, f1: 0.873, kappa: 0.867, weighted-F1: 0.873
file: checkpoint_62.json, f1: 0.873, kappa: 0.867, weighted-F1: 0.873
file: checkpoint_63.json, f1: 0.877, kappa: 0.873, weighted-F1: 0.877
file: checkpoint_64.json, f1: 0.874, kappa: 0.869, weighted-F1: 0.874
file: checkpoint_65.json, f1: 0.874, kappa: 0.869, weighted-F1: 0.874
file: checkpoint_66.json, f1: 0.873, kappa: 0.868, weighted-F1: 0.873
file: checkpoint_67.json, f1: 0.874, kappa: 0.870, weighted-F1: 0.874
file: checkpoint_68.json, f1: 0.874, kappa: 0.869, weighted-F1: 0.874
file: checkpoint_69.json, f1: 0.876, kappa: 0.872, weighted-F1: 0.876
file: checkpoint_70.json, f1: 0.874, kappa: 0.870, weighted-F1: 0.874
file: checkpoint_71.json, f1: 0.874, kappa: 0.867, weighted-F1: 0.874
file: checkpoint_72.json, f1: 0.874, kappa: 0.869, weighted-F1: 0.874
file: checkpoint_73.json, f1: 0.875, kappa: 0.868, weighted-F1: 0.875
file: checkpoint_74.json, f1: 0.875, kappa: 0.871, weighted-F1: 0.875
file: checkpoint_75.json, f1: 0.874, kappa: 0.869, weighted-F1: 0.874
file: checkpoint_76.json, f1: 0.874, kappa: 0.867, weighted-F1: 0.874
file: checkpoint_77.json, f1: 0.875, kappa: 0.870, weighted-F1: 0.875
file: checkpoint_78.json, f1: 0.873, kappa: 0.869, weighted-F1: 0.873
file: checkpoint_79.json, f1: 0.875, kappa: 0.871, weighted-F1: 0.875
file: checkpoint_80.json, f1: 0.875, kappa: 0.870, weighted-F1: 0.875
file: checkpoint_81.json, f1: 0.876, kappa: 0.876, weighted-F1: 0.876
file: checkpoint_82.json, f1: 0.876, kappa: 0.871, weighted-F1: 0.876
file: checkpoint_83.json, f1: 0.875, kappa: 0.872, weighted-F1: 0.875
file: checkpoint_84.json, f1: 0.875, kappa: 0.871, weighted-F1: 0.875
file: checkpoint_85.json, f1: 0.874, kappa: 0.867, weighted-F1: 0.874
file: checkpoint_86.json, f1: 0.874, kappa: 0.869, weighted-F1: 0.874
file: checkpoint_87.json, f1: 0.872, kappa: 0.865, weighted-F1: 0.872
file: checkpoint_88.json, f1: 0.873, kappa: 0.866, weighted-F1: 0.873
file: checkpoint_89.json, f1: 0.874, kappa: 0.868, weighted-F1: 0.874
file: checkpoint_90.json, f1: 0.875, kappa: 0.870, weighted-F1: 0.875
file: checkpoint_91.json, f1: 0.873, kappa: 0.867, weighted-F1: 0.873
file: checkpoint_92.json, f1: 0.875, kappa: 0.869, weighted-F1: 0.875
file: checkpoint_93.json, f1: 0.874, kappa: 0.867, weighted-F1: 0.874
file: checkpoint_94.json, f1: 0.874, kappa: 0.868, weighted-F1: 0.874
file: checkpoint_95.json, f1: 0.874, kappa: 0.870, weighted-F1: 0.874
file: checkpoint_96.json, f1: 0.874, kappa: 0.869, weighted-F1: 0.874
file: checkpoint_97.json, f1: 0.873, kappa: 0.868, weighted-F1: 0.873
file: checkpoint_98.json, f1: 0.875, kappa: 0.871, weighted-F1: 0.875
file: checkpoint_99.json, f1: 0.873, kappa: 0.868, weighted-F1: 0.873
file: checkpoint_100.json, f1: 0.875, kappa: 0.870, weighted-F1: 0.875
file: checkpoint_101.json, f1: 0.875, kappa: 0.871, weighted-F1: 0.875
file: checkpoint_102.json, f1: 0.874, kappa: 0.865, weighted-F1: 0.874
file: checkpoint_103.json, f1: 0.873, kappa: 0.868, weighted-F1: 0.873
file: checkpoint_104.json, f1: 0.874, kappa: 0.870, weighted-F1: 0.874
file: checkpoint_105.json, f1: 0.872, kappa: 0.865, weighted-F1: 0.872
file: checkpoint_106.json, f1: 0.874, kappa: 0.867, weighted-F1: 0.874
file: checkpoint_107.json, f1: 0.875, kappa: 0.872, weighted-F1: 0.875
file: checkpoint_108.json, f1: 0.875, kappa: 0.870, weighted-F1: 0.875
file: checkpoint_109.json, f1: 0.875, kappa: 0.870, weighted-F1: 0.875
file: checkpoint_110.json, f1: 0.875, kappa: 0.870, weighted-F1: 0.875
file: checkpoint_111.json, f1: 0.876, kappa: 0.874, weighted-F1: 0.876
file: checkpoint_112.json, f1: 0.874, kappa: 0.868, weighted-F1: 0.874
file: checkpoint_113.json, f1: 0.873, kappa: 0.868, weighted-F1: 0.873
file: checkpoint_114.json, f1: 0.872, kappa: 0.866, weighted-F1: 0.872
file: checkpoint_115.json, f1: 0.873, kappa: 0.866, weighted-F1: 0.873
file: checkpoint_116.json, f1: 0.875, kappa: 0.870, weighted-F1: 0.875
file: checkpoint_117.json, f1: 0.873, kappa: 0.865, weighted-F1: 0.873
file: checkpoint_118.json, f1: 0.874, kappa: 0.868, weighted-F1: 0.874
file: checkpoint_119.json, f1: 0.873, kappa: 0.867, weighted-F1: 0.873
file: checkpoint_120.json, f1: 0.875, kappa: 0.870, weighted-F1: 0.875
file: checkpoint_121.json, f1: 0.874, kappa: 0.869, weighted-F1: 0.874
file: checkpoint_122.json, f1: 0.874, kappa: 0.868, weighted-F1: 0.874
file: checkpoint_123.json, f1: 0.874, kappa: 0.867, weighted-F1: 0.874
file: checkpoint_124.json, f1: 0.874, kappa: 0.869, weighted-F1: 0.874
file: checkpoint_125.json, f1: 0.874, kappa: 0.869, weighted-F1: 0.874
file: checkpoint_126.json, f1: 0.874, kappa: 0.868, weighted-F1: 0.874
file: checkpoint_127.json, f1: 0.875, kappa: 0.868, weighted-F1: 0.875
file: checkpoint_128.json, f1: 0.875, kappa: 0.870, weighted-F1: 0.875
file: checkpoint_129.json, f1: 0.875, kappa: 0.867, weighted-F1: 0.875
file: checkpoint_130.json, f1: 0.875, kappa: 0.870, weighted-F1: 0.875
file: checkpoint_131.json, f1: 0.874, kappa: 0.866, weighted-F1: 0.874
file: checkpoint_132.json, f1: 0.873, kappa: 0.866, weighted-F1: 0.873
file: checkpoint_133.json, f1: 0.873, kappa: 0.866, weighted-F1: 0.873
file: checkpoint_134.json, f1: 0.873, kappa: 0.869, weighted-F1: 0.873
file: checkpoint_135.json, f1: 0.876, kappa: 0.871, weighted-F1: 0.876
file: checkpoint_136.json, f1: 0.875, kappa: 0.869, weighted-F1: 0.875
file: checkpoint_137.json, f1: 0.875, kappa: 0.872, weighted-F1: 0.875
file: checkpoint_138.json, f1: 0.875, kappa: 0.869, weighted-F1: 0.875
file: checkpoint_139.json, f1: 0.874, kappa: 0.868, weighted-F1: 0.874
file: checkpoint_140.json, f1: 0.874, kappa: 0.868, weighted-F1: 0.874
file: checkpoint_141.json, f1: 0.876, kappa: 0.869, weighted-F1: 0.876
file: checkpoint_142.json, f1: 0.874, kappa: 0.870, weighted-F1: 0.874
file: checkpoint_143.json, f1: 0.874, kappa: 0.871, weighted-F1: 0.874
file: checkpoint_144.json, f1: 0.873, kappa: 0.868, weighted-F1: 0.873
file: checkpoint_145.json, f1: 0.874, kappa: 0.869, weighted-F1: 0.874
file: checkpoint_146.json, f1: 0.873, kappa: 0.868, weighted-F1: 0.873
file: checkpoint_147.json, f1: 0.873, kappa: 0.867, weighted-F1: 0.873
file: checkpoint_148.json, f1: 0.874, kappa: 0.868, weighted-F1: 0.874
file: checkpoint_149.json, f1: 0.874, kappa: 0.869, weighted-F1: 0.874
file: checkpoint_150.json, f1: 0.874, kappa: 0.867, weighted-F1: 0.874
file: checkpoint_151.json, f1: 0.874, kappa: 0.870, weighted-F1: 0.874
file: checkpoint_152.json, f1: 0.874, kappa: 0.867, weighted-F1: 0.874
file: checkpoint_153.json, f1: 0.873, kappa: 0.864, weighted-F1: 0.873
file: checkpoint_154.json, f1: 0.874, kappa: 0.867, weighted-F1: 0.874
file: checkpoint_155.json, f1: 0.874, kappa: 0.870, weighted-F1: 0.874
file: checkpoint_156.json, f1: 0.874, kappa: 0.867, weighted-F1: 0.874
file: checkpoint_157.json, f1: 0.875, kappa: 0.869, weighted-F1: 0.875
file: checkpoint_158.json, f1: 0.874, kappa: 0.869, weighted-F1: 0.874
file: checkpoint_159.json, f1: 0.874, kappa: 0.867, weighted-F1: 0.874
file: checkpoint_160.json, f1: 0.876, kappa: 0.873, weighted-F1: 0.876
file: checkpoint_161.json, f1: 0.874, kappa: 0.868, weighted-F1: 0.874
file: checkpoint_162.json, f1: 0.876, kappa: 0.876, weighted-F1: 0.876
file: checkpoint_163.json, f1: 0.875, kappa: 0.869, weighted-F1: 0.875
file: checkpoint_164.json, f1: 0.874, kappa: 0.866, weighted-F1: 0.874
file: checkpoint_165.json, f1: 0.873, kappa: 0.866, weighted-F1: 0.873
file: checkpoint_166.json, f1: 0.875, kappa: 0.871, weighted-F1: 0.875
file: checkpoint_167.json, f1: 0.873, kappa: 0.866, weighted-F1: 0.873
file: checkpoint_168.json, f1: 0.875, kappa: 0.869, weighted-F1: 0.875
file: checkpoint_169.json, f1: 0.875, kappa: 0.870, weighted-F1: 0.875
file: checkpoint_170.json, f1: 0.873, kappa: 0.869, weighted-F1: 0.873
file: checkpoint_171.json, f1: 0.876, kappa: 0.871, weighted-F1: 0.876
file: checkpoint_172.json, f1: 0.874, kappa: 0.868, weighted-F1: 0.874
file: checkpoint_173.json, f1: 0.874, kappa: 0.868, weighted-F1: 0.874
file: checkpoint_174.json, f1: 0.875, kappa: 0.871, weighted-F1: 0.875
file: checkpoint_175.json, f1: 0.875, kappa: 0.869, weighted-F1: 0.875
file: checkpoint_176.json, f1: 0.875, kappa: 0.869, weighted-F1: 0.875
file: checkpoint_177.json, f1: 0.875, kappa: 0.871, weighted-F1: 0.875
file: checkpoint_178.json, f1: 0.873, kappa: 0.867, weighted-F1: 0.873
file: checkpoint_179.json, f1: 0.875, kappa: 0.867, weighted-F1: 0.875
file: checkpoint_180.json, f1: 0.874, kappa: 0.869, weighted-F1: 0.874
file: checkpoint_181.json, f1: 0.874, kappa: 0.868, weighted-F1: 0.874
file: checkpoint_182.json, f1: 0.877, kappa: 0.874, weighted-F1: 0.877
file: checkpoint_183.json, f1: 0.875, kappa: 0.871, weighted-F1: 0.875
file: checkpoint_184.json, f1: 0.874, kappa: 0.868, weighted-F1: 0.874
file: checkpoint_185.json, f1: 0.875, kappa: 0.868, weighted-F1: 0.875
file: checkpoint_186.json, f1: 0.874, kappa: 0.868, weighted-F1: 0.874
file: checkpoint_187.json, f1: 0.876, kappa: 0.872, weighted-F1: 0.876
file: checkpoint_188.json, f1: 0.872, kappa: 0.866, weighted-F1: 0.872
file: checkpoint_189.json, f1: 0.875, kappa: 0.868, weighted-F1: 0.875
file: checkpoint_190.json, f1: 0.875, kappa: 0.870, weighted-F1: 0.875
file: checkpoint_191.json, f1: 0.875, kappa: 0.871, weighted-F1: 0.875
file: checkpoint_192.json, f1: 0.874, kappa: 0.867, weighted-F1: 0.874
file: checkpoint_193.json, f1: 0.873, kappa: 0.867, weighted-F1: 0.873
file: checkpoint_194.json, f1: 0.874, kappa: 0.869, weighted-F1: 0.874
file: checkpoint_195.json, f1: 0.873, kappa: 0.867, weighted-F1: 0.873
file: checkpoint_196.json, f1: 0.875, kappa: 0.870, weighted-F1: 0.875
file: checkpoint_197.json, f1: 0.876, kappa: 0.872, weighted-F1: 0.876
file: checkpoint_198.json, f1: 0.874, kappa: 0.869, weighted-F1: 0.874
file: checkpoint_199.json, f1: 0.873, kappa: 0.869, weighted-F1: 0.873
file: checkpoint_200.json, f1: 0.875, kappa: 0.870, weighted-F1: 0.875
Out[9]:
([0.87525038267402089, 0.8749483190850349, 0.87490132863993875],
 [array([[894,   4,  30,  12,   9,   0,   2,   5,  39,   5],
         [ 11, 949,   3,   4,   0,   0,   5,   1,  19,   8],
         [ 31,   0, 884,  20,  31,  10,  17,   2,   5,   0],
         [  9,   2,  47, 798,  26,  67,  39,   8,   3,   1],
         [  5,   1,  40,  28, 877,  10,  20,  16,   3,   0],
         [  2,   0,  42, 165,  24, 746,   8,  13,   0,   0],
         [  4,   1,  42,  33,   9,   4, 906,   0,   1,   0],
         [  5,   0,  27,  34,  26,  17,   0, 890,   1,   0],
         [ 28,   3,  10,   3,   4,   0,   2,   0, 947,   3],
         [ 24,  76,   4,   8,   3,   0,   4,   2,  23, 856]]),
  array([[900,   2,  27,  13,   9,   0,   2,   5,  37,   5],
         [ 12, 941,   3,   6,   0,   0,   6,   1,  21,  10],
         [ 33,   0, 881,  22,  30,  11,  16,   2,   5,   0],
         [  9,   1,  46, 801,  23,  68,  39,   9,   3,   1],
         [  7,   1,  43,  32, 868,  10,  19,  18,   2,   0],
         [  2,   0,  40, 167,  23, 746,   7,  15,   0,   0],
         [  4,   1,  42,  40,   8,   5, 899,   0,   1,   0],
         [  4,   0,  25,  33,  23,  17,   0, 897,   1,   0],
         [ 29,   3,  11,   3,   4,   0,   1,   0, 946,   3],
         [ 25,  65,   5,  10,   1,   1,   4,   2,  24, 863]]),
  array([[901,   4,  26,  16,   7,   0,   2,   4,  35,   5],
         [ 12, 949,   3,   5,   0,   0,   5,   1,  17,   8],
         [ 35,   0, 871,  27,  31,  13,  15,   3,   5,   0],
         [ 10,   2,  45, 808,  23,  63,  38,   7,   3,   1],
         [  7,   1,  41,  33, 870,  10,  19,  17,   2,   0],
         [  2,   0,  37, 171,  22, 747,   7,  14,   0,   0],
         [  4,   1,  41,  42,   7,   5, 899,   0,   1,   0],
         [  6,   0,  23,  38,  25,  18,   0, 889,   1,   0],
         [ 29,   4,  12,   3,   4,   0,   1,   0, 943,   4],
         [ 24,  72,   4,  10,   2,   0,   3,   2,  20, 863]])],
 ['checkpoint_185.json', 'checkpoint_196.json', 'checkpoint_200.json'])